{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "898c5a7e",
   "metadata": {},
   "source": [
    "------\n",
    "* Python Web crawler\n",
    "* 2023.10.25\n",
    "* week08:猎聘系统性抓取\n",
    "------"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "46a70dd3",
   "metadata": {},
   "source": [
    "# 请求数据准备"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "0a02205e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Requirement already satisfied: requests-html in /Users/kristenleung/opt/anaconda3/lib/python3.9/site-packages (0.10.0)\n",
      "Requirement already satisfied: pyppeteer>=0.0.14 in /Users/kristenleung/opt/anaconda3/lib/python3.9/site-packages (from requests-html) (1.0.2)\n",
      "Requirement already satisfied: w3lib in /Users/kristenleung/opt/anaconda3/lib/python3.9/site-packages (from requests-html) (1.21.0)\n",
      "Requirement already satisfied: fake-useragent in /Users/kristenleung/opt/anaconda3/lib/python3.9/site-packages (from requests-html) (1.2.1)\n",
      "Requirement already satisfied: requests in /Users/kristenleung/opt/anaconda3/lib/python3.9/site-packages (from requests-html) (2.27.1)\n",
      "Requirement already satisfied: bs4 in /Users/kristenleung/opt/anaconda3/lib/python3.9/site-packages (from requests-html) (0.0.1)\n",
      "Requirement already satisfied: parse in /Users/kristenleung/opt/anaconda3/lib/python3.9/site-packages (from requests-html) (1.19.1)\n",
      "Requirement already satisfied: pyquery in /Users/kristenleung/opt/anaconda3/lib/python3.9/site-packages (from requests-html) (2.0.0)\n",
      "Requirement already satisfied: importlib-metadata>=1.4 in /Users/kristenleung/opt/anaconda3/lib/python3.9/site-packages (from pyppeteer>=0.0.14->requests-html) (4.11.3)\n",
      "Requirement already satisfied: pyee<9.0.0,>=8.1.0 in /Users/kristenleung/opt/anaconda3/lib/python3.9/site-packages (from pyppeteer>=0.0.14->requests-html) (8.2.2)\n",
      "Requirement already satisfied: certifi>=2021 in /Users/kristenleung/opt/anaconda3/lib/python3.9/site-packages (from pyppeteer>=0.0.14->requests-html) (2021.10.8)\n",
      "Requirement already satisfied: tqdm<5.0.0,>=4.42.1 in /Users/kristenleung/opt/anaconda3/lib/python3.9/site-packages (from pyppeteer>=0.0.14->requests-html) (4.64.0)\n",
      "Requirement already satisfied: urllib3<2.0.0,>=1.25.8 in /Users/kristenleung/opt/anaconda3/lib/python3.9/site-packages (from pyppeteer>=0.0.14->requests-html) (1.26.9)\n",
      "Requirement already satisfied: websockets<11.0,>=10.0 in /Users/kristenleung/opt/anaconda3/lib/python3.9/site-packages (from pyppeteer>=0.0.14->requests-html) (10.4)\n",
      "Requirement already satisfied: appdirs<2.0.0,>=1.4.3 in /Users/kristenleung/opt/anaconda3/lib/python3.9/site-packages (from pyppeteer>=0.0.14->requests-html) (1.4.4)\n",
      "Requirement already satisfied: zipp>=0.5 in /Users/kristenleung/opt/anaconda3/lib/python3.9/site-packages (from importlib-metadata>=1.4->pyppeteer>=0.0.14->requests-html) (3.7.0)\n",
      "Requirement already satisfied: beautifulsoup4 in /Users/kristenleung/opt/anaconda3/lib/python3.9/site-packages (from bs4->requests-html) (4.11.1)\n",
      "Requirement already satisfied: soupsieve>1.2 in /Users/kristenleung/opt/anaconda3/lib/python3.9/site-packages (from beautifulsoup4->bs4->requests-html) (2.3.1)\n",
      "Requirement already satisfied: importlib-resources>=5.0 in /Users/kristenleung/opt/anaconda3/lib/python3.9/site-packages (from fake-useragent->requests-html) (6.0.1)\n",
      "Requirement already satisfied: cssselect>=1.2.0 in /Users/kristenleung/opt/anaconda3/lib/python3.9/site-packages (from pyquery->requests-html) (1.2.0)\n",
      "Requirement already satisfied: lxml>=2.1 in /Users/kristenleung/opt/anaconda3/lib/python3.9/site-packages (from pyquery->requests-html) (4.8.0)\n",
      "Requirement already satisfied: charset-normalizer~=2.0.0 in /Users/kristenleung/opt/anaconda3/lib/python3.9/site-packages (from requests->requests-html) (2.0.4)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /Users/kristenleung/opt/anaconda3/lib/python3.9/site-packages (from requests->requests-html) (3.3)\n",
      "Requirement already satisfied: six>=1.4.1 in /Users/kristenleung/opt/anaconda3/lib/python3.9/site-packages (from w3lib->requests-html) (1.16.0)\n"
     ]
    }
   ],
   "source": [
    "!pip install requests-html"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "98db56be",
   "metadata": {},
   "outputs": [],
   "source": [
    "# header\n",
    "import liepindata\n",
    "headers = liepindata.headers()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "d231d6e3",
   "metadata": {},
   "outputs": [
    {
     "ename": "TypeError",
     "evalue": "request_payload() got an unexpected keyword argument '公司规模'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "Input \u001b[0;32mIn [10]\u001b[0m, in \u001b[0;36m<cell line: 2>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[38;5;66;03m# payload\u001b[39;00m\n\u001b[0;32m----> 2\u001b[0m payload \u001b[38;5;241m=\u001b[39m \u001b[43mliepindata\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mrequest_payload\u001b[49m\u001b[43m(\u001b[49m\u001b[43m城市\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43m深圳\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m关键词\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43m数据分析师\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m工作经验\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43m实习生\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m公司规模\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43m10000人以上\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m)\u001b[49m\n\u001b[1;32m      3\u001b[0m payload\n",
      "\u001b[0;31mTypeError\u001b[0m: request_payload() got an unexpected keyword argument '公司规模'"
     ]
    }
   ],
   "source": [
    "# payload\n",
    "payload = liepindata.request_payload(城市 = \"深圳\", 关键词 = \"数据分析师\", 工作经验=\"实习生\", 公司规模 = \"10000人以上\")\n",
    "payload"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "f7ae1b30",
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'r' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "Input \u001b[0;32mIn [7]\u001b[0m, in \u001b[0;36m<cell line: 1>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0m liepin_data \u001b[38;5;241m=\u001b[39m\u001b[43mr\u001b[49m\u001b[38;5;241m.\u001b[39mjson()\n\u001b[1;32m      2\u001b[0m liepin_data\n",
      "\u001b[0;31mNameError\u001b[0m: name 'r' is not defined"
     ]
    }
   ],
   "source": [
    "liepin_data =r.json()\n",
    "liepin_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "b0927da7",
   "metadata": {},
   "outputs": [
    {
     "ename": "SyntaxError",
     "evalue": "invalid syntax (365955700.py, line 7)",
     "output_type": "error",
     "traceback": [
      "\u001b[0;36m  Input \u001b[0;32mIn [11]\u001b[0;36m\u001b[0m\n\u001b[0;31m    totalPage payload\u001b[0m\n\u001b[0m              ^\u001b[0m\n\u001b[0;31mSyntaxError\u001b[0m\u001b[0;31m:\u001b[0m invalid syntax\n"
     ]
    }
   ],
   "source": [
    "#思考：页码的范围如何确定？\n",
    "# Totalpage决定 请求的内容有多少页面\n",
    "# totalpage 至少需要先请求一次才能确定\n",
    "\n",
    "\n",
    "payload = liepindata.request_payload(城市 = \"深圳\", 关键词 = \"产品经理\", 工作经验=\"\", 公司规模 = \"\")\n",
    "totalPage payload"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "96f6c69f",
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'payload' is not defined",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "Input \u001b[0;32mIn [12]\u001b[0m, in \u001b[0;36m<cell line: 5>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mrequests_html\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m HTMLSession\n\u001b[1;32m      4\u001b[0m session \u001b[38;5;241m=\u001b[39m HTMLSession()\n\u001b[1;32m      5\u001b[0m r \u001b[38;5;241m=\u001b[39m session\u001b[38;5;241m.\u001b[39mpost(\n\u001b[1;32m      6\u001b[0m             url \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m'\u001b[39m\u001b[38;5;124mhttps://api-c.liepin.com/api/com.liepin.searchfront4c.pc-search-job\u001b[39m\u001b[38;5;124m'\u001b[39m,\n\u001b[0;32m----> 7\u001b[0m             data \u001b[38;5;241m=\u001b[39m json\u001b[38;5;241m.\u001b[39mdumps(\u001b[43mpayload\u001b[49m), \u001b[38;5;66;03m# POST请求带的参数\u001b[39;00m\n\u001b[1;32m      8\u001b[0m             headers \u001b[38;5;241m=\u001b[39m headers    \u001b[38;5;66;03m# 模拟真实用户\u001b[39;00m\n\u001b[1;32m      9\u001b[0m )\n",
      "\u001b[0;31mNameError\u001b[0m: name 'payload' is not defined"
     ]
    }
   ],
   "source": [
    "import json\n",
    "from requests_html import HTMLSession\n",
    "\n",
    "session = HTMLSession()\n",
    "r = session.post(\n",
    "            url = 'https://api-c.liepin.com/api/com.liepin.searchfront4c.pc-search-job',\n",
    "            data = json.dumps(payload), # POST请求带的参数\n",
    "            headers = headers    # 模拟真实用户\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f67dd99f",
   "metadata": {},
   "outputs": [],
   "source": [
    "result = r.json()\n",
    "result"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d5a0ebe1",
   "metadata": {},
   "outputs": [],
   "source": [
    "totalPage = r.json()['data']['pagination']['totalPage']\n",
    "totalPage"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "70e0a12f",
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "from requests_html import HTMLSession\n",
    "import time \n",
    "import random\n",
    "\n",
    "## 综上所述\n",
    "# crawler\n",
    "crawl_liepin_data = []\n",
    "\n",
    "# 1.确定页码数量\n",
    "payload = liepindata.request_payload(城市 = \"深圳\", 关键词 = \"产品经理\", 工作经验=\"\", 公司规模 = \"\")\n",
    "r = session.post(\n",
    "            url = 'https://api-c.liepin.com/api/com.liepin.searchfront4c.pc-search-job',\n",
    "            data = json.dumps(payload), # POST请求带的参数\n",
    "            headers = headers    # 模拟真实用户\n",
    ")\n",
    "totalPage = r.json()['data']['pagination']['totalPage']\n",
    "\n",
    "# 2.翻页请求（按照上述获取到的页面数量进行翻页请求）\n",
    "for i in range(totalPage):\n",
    "    wait_time = radom.randint(3,10)\n",
    "    # 1.翻页请求的准备工作\n",
    "    payload = liepindata.request_payload(城市 = \"深圳\", 关键词 = \"数据分析师\", 工作经验=\"实习生\", 公司规模 = \"10000人以上\")\n",
    "    print(payload)\n",
    "    # 2.发起请求\n",
    "    r = session.post(\n",
    "            url = 'https://api-c.liepin.com/api/com.liepin.searchfront4c.pc-search-job',\n",
    "            data = json.dumps(payload), # POST请求带的参数\n",
    "            headers = headers    # 模拟真实用户\n",
    "        )\n",
    "    print(\"这是第%s页，接下来将先等待%s秒...然后以继续抓取\"%(i+1,wait_time))\n",
    "#     print(r.json())\n",
    "    crawl_liepin_data.append(r.json())\n",
    "    # 3.分批次发起等待时间\n",
    "    time.sleep(wait_time)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5d8767c6",
   "metadata": {},
   "outputs": [],
   "source": [
    "crawl_liepin_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2a1abde4",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "pd.json_normalize(crawl_liepin_data[0]['data']['data']['jobCardList'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e5f8306d",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "25661ad1",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1c7f553e",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
